{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "88e273d4b5d4f8cd",
   "metadata": {
    "id": "88e273d4b5d4f8cd"
   },
   "source": [
    "# 下面是用CLIP模型做的推理"
   ]
  },
  {
   "cell_type": "code",
   "source": [
    "!pip list|grep torch"
   ],
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "mVDmcONCDeYe",
    "outputId": "f9c2ce4d-406d-4c08-b41a-f2a9fc730032",
    "ExecuteTime": {
     "end_time": "2025-08-04T02:05:29.220723Z",
     "start_time": "2025-08-04T02:05:26.618709Z"
    }
   },
   "id": "mVDmcONCDeYe",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch                     2.7.1+cu126\n",
      "torchaudio                2.7.1+cu126\n",
      "torchvision               0.22.1+cu126\n",
      "torchviz                  0.0.3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "[notice] A new release of pip is available: 24.0 -> 25.2\n",
      "[notice] To update, run: python.exe -m pip install --upgrade pip\n"
     ]
    }
   ],
   "execution_count": 1
  },
  {
   "cell_type": "code",
   "source": [
    "!pip list|grep transformers"
   ],
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "yc1PjEChF-Om",
    "outputId": "bcc9d447-7006-46d2-befb-69d645204102",
    "ExecuteTime": {
     "end_time": "2025-08-04T02:06:05.396555Z",
     "start_time": "2025-08-04T02:06:03.782858Z"
    }
   },
   "id": "yc1PjEChF-Om",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sentence-transformers     5.0.0\n",
      "transformers              4.53.3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "[notice] A new release of pip is available: 24.0 -> 25.2\n",
      "[notice] To update, run: python.exe -m pip install --upgrade pip\n"
     ]
    }
   ],
   "execution_count": 2
  },
  {
   "cell_type": "code",
   "id": "initial_id",
   "metadata": {
    "collapsed": true,
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "initial_id",
    "outputId": "59ff18d8-2aa8-4ee5-f21a-7ce7b924f936",
    "ExecuteTime": {
     "end_time": "2025-08-04T02:09:56.853295Z",
     "start_time": "2025-08-04T02:09:52.900183Z"
    }
   },
   "source": [
    "from transformers import CLIPProcessor, CLIPModel\n",
    "from PIL import Image\n",
    "\n",
    "\n",
    "# 加载预训练的 CLIP 模型和处理器\n",
    "model = CLIPModel.from_pretrained(\"openai/clip-vit-base-patch32\")\n",
    "processor = CLIPProcessor.from_pretrained(\"openai/clip-vit-base-patch32\")\n",
    "print(model)\n",
    "print('-'*100)\n",
    "print(processor)\n",
    "# 加载图像\n",
    "image = Image.open('luke.jpg')\n",
    "\n",
    "# 定义文本描述\n",
    "texts = [\"a photo of a cat\", \"a photo of a dog\", \"a photo of a girl\"]\n",
    "\n",
    "# 处理输入\n",
    "inputs = processor(text=texts, images=image, return_tensors=\"pt\", padding=True)\n",
    "print(inputs)\n",
    "# 前向传播\n",
    "outputs = model(**inputs)\n",
    "logits_per_image = outputs.logits_per_image  # 图像与文本的相似度\n",
    "probs = logits_per_image.softmax(dim=1)  # 转换为概率\n",
    "\n",
    "# 输出结果\n",
    "print(\"Probabilities:\", probs)"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CLIPModel(\n",
      "  (text_model): CLIPTextTransformer(\n",
      "    (embeddings): CLIPTextEmbeddings(\n",
      "      (token_embedding): Embedding(49408, 512)\n",
      "      (position_embedding): Embedding(77, 512)\n",
      "    )\n",
      "    (encoder): CLIPEncoder(\n",
      "      (layers): ModuleList(\n",
      "        (0-11): 12 x CLIPEncoderLayer(\n",
      "          (self_attn): CLIPAttention(\n",
      "            (k_proj): Linear(in_features=512, out_features=512, bias=True)\n",
      "            (v_proj): Linear(in_features=512, out_features=512, bias=True)\n",
      "            (q_proj): Linear(in_features=512, out_features=512, bias=True)\n",
      "            (out_proj): Linear(in_features=512, out_features=512, bias=True)\n",
      "          )\n",
      "          (layer_norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "          (mlp): CLIPMLP(\n",
      "            (activation_fn): QuickGELUActivation()\n",
      "            (fc1): Linear(in_features=512, out_features=2048, bias=True)\n",
      "            (fc2): Linear(in_features=2048, out_features=512, bias=True)\n",
      "          )\n",
      "          (layer_norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (final_layer_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "  )\n",
      "  (vision_model): CLIPVisionTransformer(\n",
      "    (embeddings): CLIPVisionEmbeddings(\n",
      "      (patch_embedding): Conv2d(3, 768, kernel_size=(32, 32), stride=(32, 32), bias=False)\n",
      "      (position_embedding): Embedding(50, 768)\n",
      "    )\n",
      "    (pre_layrnorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "    (encoder): CLIPEncoder(\n",
      "      (layers): ModuleList(\n",
      "        (0-11): 12 x CLIPEncoderLayer(\n",
      "          (self_attn): CLIPAttention(\n",
      "            (k_proj): Linear(in_features=768, out_features=768, bias=True)\n",
      "            (v_proj): Linear(in_features=768, out_features=768, bias=True)\n",
      "            (q_proj): Linear(in_features=768, out_features=768, bias=True)\n",
      "            (out_proj): Linear(in_features=768, out_features=768, bias=True)\n",
      "          )\n",
      "          (layer_norm1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "          (mlp): CLIPMLP(\n",
      "            (activation_fn): QuickGELUActivation()\n",
      "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
      "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
      "          )\n",
      "          (layer_norm2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (post_layernorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "  )\n",
      "  (visual_projection): Linear(in_features=768, out_features=512, bias=False)\n",
      "  (text_projection): Linear(in_features=512, out_features=512, bias=False)\n",
      ")\n",
      "----------------------------------------------------------------------------------------------------\n",
      "CLIPProcessor:\n",
      "- image_processor: CLIPImageProcessor {\n",
      "  \"crop_size\": {\n",
      "    \"height\": 224,\n",
      "    \"width\": 224\n",
      "  },\n",
      "  \"do_center_crop\": true,\n",
      "  \"do_convert_rgb\": true,\n",
      "  \"do_normalize\": true,\n",
      "  \"do_rescale\": true,\n",
      "  \"do_resize\": true,\n",
      "  \"image_mean\": [\n",
      "    0.48145466,\n",
      "    0.4578275,\n",
      "    0.40821073\n",
      "  ],\n",
      "  \"image_processor_type\": \"CLIPImageProcessor\",\n",
      "  \"image_std\": [\n",
      "    0.26862954,\n",
      "    0.26130258,\n",
      "    0.27577711\n",
      "  ],\n",
      "  \"resample\": 3,\n",
      "  \"rescale_factor\": 0.00392156862745098,\n",
      "  \"size\": {\n",
      "    \"shortest_edge\": 224\n",
      "  }\n",
      "}\n",
      "\n",
      "- tokenizer: CLIPTokenizerFast(name_or_path='openai/clip-vit-base-patch32', vocab_size=49408, model_max_length=77, is_fast=True, padding_side='right', truncation_side='right', special_tokens={'bos_token': '<|startoftext|>', 'eos_token': '<|endoftext|>', 'unk_token': '<|endoftext|>', 'pad_token': '<|endoftext|>'}, clean_up_tokenization_spaces=False, added_tokens_decoder={\n",
      "\t49406: AddedToken(\"<|startoftext|>\", rstrip=False, lstrip=False, single_word=False, normalized=True, special=True),\n",
      "\t49407: AddedToken(\"<|endoftext|>\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
      "}\n",
      ")\n",
      "\n",
      "{\n",
      "  \"processor_class\": \"CLIPProcessor\"\n",
      "}\n",
      "\n",
      "{'input_ids': tensor([[49406,   320,  1125,   539,   320,  2368, 49407],\n",
      "        [49406,   320,  1125,   539,   320,  1929, 49407],\n",
      "        [49406,   320,  1125,   539,   320,  1611, 49407]]), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1],\n",
      "        [1, 1, 1, 1, 1, 1, 1],\n",
      "        [1, 1, 1, 1, 1, 1, 1]]), 'pixel_values': tensor([[[[1.5070, 1.5070, 1.5070,  ..., 1.5070, 1.5070, 1.5070],\n",
      "          [1.5070, 1.5070, 1.5070,  ..., 1.5070, 1.5070, 1.5070],\n",
      "          [1.5070, 1.5070, 1.5070,  ..., 1.5070, 1.5070, 1.5070],\n",
      "          ...,\n",
      "          [1.5070, 1.5070, 1.5070,  ..., 1.2880, 1.5508, 1.4340],\n",
      "          [1.5070, 1.5070, 1.5070,  ..., 0.8938, 1.6238, 1.5362],\n",
      "          [1.5070, 1.5070, 1.5070,  ..., 1.1712, 1.3026, 1.4486]],\n",
      "\n",
      "         [[1.6397, 1.6397, 1.6397,  ..., 1.6397, 1.6397, 1.6397],\n",
      "          [1.6397, 1.6397, 1.6397,  ..., 1.6397, 1.6397, 1.6397],\n",
      "          [1.6397, 1.6397, 1.6397,  ..., 1.6397, 1.6397, 1.6397],\n",
      "          ...,\n",
      "          [1.6397, 1.6397, 1.6397,  ..., 1.2495, 1.6847, 1.5346],\n",
      "          [1.6397, 1.6397, 1.6397,  ..., 0.5591, 1.7297, 1.6247],\n",
      "          [1.6397, 1.6397, 1.6397,  ..., 1.0093, 1.3395, 1.6247]],\n",
      "\n",
      "         [[1.7335, 1.7335, 1.7335,  ..., 1.7335, 1.7335, 1.7335],\n",
      "          [1.7335, 1.7335, 1.7335,  ..., 1.7335, 1.7335, 1.7335],\n",
      "          [1.7335, 1.7335, 1.7335,  ..., 1.7335, 1.7335, 1.7335],\n",
      "          ...,\n",
      "          [1.7335, 1.7335, 1.7335,  ..., 0.3399, 1.2927, 1.0083],\n",
      "          [1.7335, 1.7335, 1.7335,  ..., 0.0129, 1.4349, 1.2500],\n",
      "          [1.7335, 1.7335, 1.7335,  ..., 0.4253, 0.9372, 1.3354]]]])}\n",
      "Probabilities: tensor([[0.0014, 0.0018, 0.9968]], grad_fn=<SoftmaxBackward0>)\n"
     ]
    }
   ],
   "execution_count": 4
  },
  {
   "cell_type": "markdown",
   "id": "df6ae718c5ea1f94",
   "metadata": {
    "id": "df6ae718c5ea1f94"
   },
   "source": []
  },
  {
   "cell_type": "code",
   "source": [
    "from transformers import CLIPProcessor, CLIPModel\n",
    "from PIL import Image\n",
    "import torch\n",
    "\n",
    "# 加载预训练的 CLIP 模型和处理器\n",
    "model = CLIPModel.from_pretrained(\"openai/clip-vit-base-patch32\")\n",
    "processor = CLIPProcessor.from_pretrained(\"openai/clip-vit-base-patch32\")\n",
    "\n",
    "print(\"CLIP Model Architecture:\")\n",
    "print(model)\n",
    "print('-'*100)\n",
    "# print(\"CLIP Processor:\")\n",
    "# print(processor)\n",
    "print('='*100)\n",
    "\n",
    "# 加载图像\n",
    "image = Image.open('luke.jpg')\n",
    "\n",
    "# 定义文本描述\n",
    "texts = [\"a photo of a cat\", \"a photo of a dog\", \"a photo of a girl\"]\n",
    "\n",
    "# 处理输入\n",
    "inputs = processor(text=texts, images=image, return_tensors=\"pt\", padding=True)\n",
    "print(\"Processed Inputs:\")\n",
    "print(f\"Input keys: {list(inputs.keys())}\")\n",
    "for key, value in inputs.items():\n",
    "    if isinstance(value, torch.Tensor):\n",
    "        print(f\"  {key}: {value.shape}\")\n",
    "    else:\n",
    "        print(f\"  {key}: {type(value)}\")\n",
    "print('-'*50)\n",
    "\n",
    "# 设置模型为评估模式\n",
    "model.eval()\n",
    "\n",
    "# 使用torch.no_grad()进行推理以节省内存\n",
    "with torch.no_grad():\n",
    "    print(\"Forward Pass Analysis:\")\n",
    "    print('-'*50)\n",
    "\n",
    "    # 分别处理文本和图像以获得中间结果\n",
    "\n",
    "    # 1. 文本编码\n",
    "    print(\"1. Text Encoding:\")\n",
    "    text_inputs = {\n",
    "        'input_ids': inputs['input_ids'],\n",
    "        'attention_mask': inputs['attention_mask']\n",
    "    }\n",
    "    print(f\"   Text input_ids shape: {text_inputs['input_ids'].shape}\")\n",
    "    print(f\"   Text attention_mask shape: {text_inputs['attention_mask'].shape}\")\n",
    "\n",
    "    # 通过text_model获取文本特征\n",
    "    text_outputs = model.text_model(**text_inputs)\n",
    "    text_embeds = model.text_projection(text_outputs.pooler_output)\n",
    "\n",
    "    print(f\"   Text model hidden states shape: {text_outputs.last_hidden_state.shape}\")\n",
    "    print(f\"   Text model pooler output shape: {text_outputs.pooler_output.shape}\")\n",
    "    print(f\"   Text embeddings (after projection) shape: {text_embeds.shape}\")\n",
    "    print()\n",
    "\n",
    "    # 2. 图像编码\n",
    "    print(\"2. Vision Encoding:\")\n",
    "    vision_inputs = {\n",
    "        'pixel_values': inputs['pixel_values']\n",
    "    }\n",
    "    print(f\"   Vision pixel_values shape: {vision_inputs['pixel_values'].shape}\")\n",
    "\n",
    "    # 通过vision_model获取图像特征\n",
    "    vision_outputs = model.vision_model(**vision_inputs)\n",
    "    image_embeds = model.visual_projection(vision_outputs.pooler_output)\n",
    "\n",
    "    print(f\"   Vision model hidden states shape: {vision_outputs.last_hidden_state.shape}\")\n",
    "    print(f\"   Vision model pooler output shape: {vision_outputs.pooler_output.shape}\")\n",
    "    print(f\"   Image embeddings (after projection) shape: {image_embeds.shape}\")\n",
    "    print()\n",
    "\n",
    "    # 3. 完整前向传播\n",
    "    print(\"3. Complete Forward Pass:\")\n",
    "    outputs = model(**inputs)\n",
    "\n",
    "    print(f\"   Final text features shape: {outputs.text_embeds.shape}\")\n",
    "    print(f\"   Final image features shape: {outputs.image_embeds.shape}\")\n",
    "    print(f\"   Logits per image shape: {outputs.logits_per_image.shape}\")\n",
    "    print(f\"   Logits per text shape: {outputs.logits_per_text.shape}\")\n",
    "    print()\n",
    "\n",
    "    # 4. 相似度计算详解\n",
    "    print(\"4. Similarity Computation Details:\")\n",
    "    logits_per_image = outputs.logits_per_image\n",
    "    logits_per_text = outputs.logits_per_text\n",
    "\n",
    "    print(f\"   Image-to-text logits: {logits_per_image.shape}\")\n",
    "    print(f\"   Text-to-image logits: {logits_per_text.shape}\")\n",
    "    print(f\"   Logit scale parameter: {model.logit_scale.exp().item():.4f}\")\n",
    "\n",
    "    # 转换为概率\n",
    "    probs_image_to_text = logits_per_image.softmax(dim=1)\n",
    "    probs_text_to_image = logits_per_text.softmax(dim=0)\n",
    "\n",
    "    print(f\"   Image-to-text probabilities shape: {probs_image_to_text.shape}\")\n",
    "    print(f\"   Text-to-image probabilities shape: {probs_text_to_image.shape}\")\n",
    "    print()\n",
    "\n",
    "    # 5. 详细结果展示\n",
    "    print(\"5. Detailed Results:\")\n",
    "    print('-'*30)\n",
    "\n",
    "    # 原始相似度分数\n",
    "    print(\"Raw similarity scores (logits):\")\n",
    "    for i, text in enumerate(texts):\n",
    "        print(f\"   '{text}': {logits_per_image[0, i].item():.4f}\")\n",
    "    print()\n",
    "\n",
    "    # 概率分布\n",
    "    print(\"Probability distribution:\")\n",
    "    probs = probs_image_to_text[0]  # 取第一张（也是唯一一张）图像的概率\n",
    "    for i, text in enumerate(texts):\n",
    "        print(f\"   '{text}': {probs[i].item():.4f} ({probs[i].item()*100:.2f}%)\")\n",
    "    print()\n",
    "\n",
    "    # 最匹配的文本\n",
    "    best_match_idx = torch.argmax(probs).item()\n",
    "    print(f\"Best match: '{texts[best_match_idx]}' with probability {probs[best_match_idx].item():.4f}\")\n",
    "    print()\n",
    "\n",
    "# 6. 模型参数统计\n",
    "print(\"6. Model Parameter Statistics:\")\n",
    "print('-'*40)\n",
    "total_params = sum(p.numel() for p in model.parameters())\n",
    "trainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad)\n",
    "\n",
    "print(f\"Total parameters: {total_params:,}\")\n",
    "print(f\"Trainable parameters: {trainable_params:,}\")\n",
    "\n",
    "# 各组件参数统计\n",
    "text_params = sum(p.numel() for p in model.text_model.parameters())\n",
    "vision_params = sum(p.numel() for p in model.vision_model.parameters())\n",
    "text_proj_params = sum(p.numel() for p in model.text_projection.parameters())\n",
    "visual_proj_params = sum(p.numel() for p in model.visual_projection.parameters())\n",
    "\n",
    "print(f\"Text model parameters: {text_params:,}\")\n",
    "print(f\"Vision model parameters: {vision_params:,}\")\n",
    "print(f\"Text projection parameters: {text_proj_params:,}\")\n",
    "print(f\"Visual projection parameters: {visual_proj_params:,}\")\n",
    "print(f\"Logit scale parameter: 1\")\n",
    "\n",
    "print('='*100)\n",
    "print(\"Analysis Complete!\")"
   ],
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "4ROsoLekNOpl",
    "outputId": "219e1060-3137-4509-aa86-e5d4fecbe8dc",
    "ExecuteTime": {
     "end_time": "2025-08-06T02:12:01.459557Z",
     "start_time": "2025-08-06T02:11:34.337973Z"
    }
   },
   "id": "4ROsoLekNOpl",
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using a slow image processor as `use_fast` is unset and a slow processor was saved with this model. `use_fast=True` will be the default behavior in v4.52, even if the model was saved with a slow processor. This will result in minor differences in outputs. You'll still be able to use a slow processor with `use_fast=False`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CLIP Model Architecture:\n",
      "CLIPModel(\n",
      "  (text_model): CLIPTextTransformer(\n",
      "    (embeddings): CLIPTextEmbeddings(\n",
      "      (token_embedding): Embedding(49408, 512)\n",
      "      (position_embedding): Embedding(77, 512)\n",
      "    )\n",
      "    (encoder): CLIPEncoder(\n",
      "      (layers): ModuleList(\n",
      "        (0-11): 12 x CLIPEncoderLayer(\n",
      "          (self_attn): CLIPAttention(\n",
      "            (k_proj): Linear(in_features=512, out_features=512, bias=True)\n",
      "            (v_proj): Linear(in_features=512, out_features=512, bias=True)\n",
      "            (q_proj): Linear(in_features=512, out_features=512, bias=True)\n",
      "            (out_proj): Linear(in_features=512, out_features=512, bias=True)\n",
      "          )\n",
      "          (layer_norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "          (mlp): CLIPMLP(\n",
      "            (activation_fn): QuickGELUActivation()\n",
      "            (fc1): Linear(in_features=512, out_features=2048, bias=True)\n",
      "            (fc2): Linear(in_features=2048, out_features=512, bias=True)\n",
      "          )\n",
      "          (layer_norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (final_layer_norm): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
      "  )\n",
      "  (vision_model): CLIPVisionTransformer(\n",
      "    (embeddings): CLIPVisionEmbeddings(\n",
      "      (patch_embedding): Conv2d(3, 768, kernel_size=(32, 32), stride=(32, 32), bias=False)\n",
      "      (position_embedding): Embedding(50, 768)\n",
      "    )\n",
      "    (pre_layrnorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "    (encoder): CLIPEncoder(\n",
      "      (layers): ModuleList(\n",
      "        (0-11): 12 x CLIPEncoderLayer(\n",
      "          (self_attn): CLIPAttention(\n",
      "            (k_proj): Linear(in_features=768, out_features=768, bias=True)\n",
      "            (v_proj): Linear(in_features=768, out_features=768, bias=True)\n",
      "            (q_proj): Linear(in_features=768, out_features=768, bias=True)\n",
      "            (out_proj): Linear(in_features=768, out_features=768, bias=True)\n",
      "          )\n",
      "          (layer_norm1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "          (mlp): CLIPMLP(\n",
      "            (activation_fn): QuickGELUActivation()\n",
      "            (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
      "            (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
      "          )\n",
      "          (layer_norm2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (post_layernorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "  )\n",
      "  (visual_projection): Linear(in_features=768, out_features=512, bias=False)\n",
      "  (text_projection): Linear(in_features=512, out_features=512, bias=False)\n",
      ")\n",
      "----------------------------------------------------------------------------------------------------\n",
      "====================================================================================================\n",
      "Processed Inputs:\n",
      "Input keys: ['input_ids', 'attention_mask', 'pixel_values']\n",
      "  input_ids: torch.Size([3, 7])\n",
      "  attention_mask: torch.Size([3, 7])\n",
      "  pixel_values: torch.Size([1, 3, 224, 224])\n",
      "--------------------------------------------------\n",
      "Forward Pass Analysis:\n",
      "--------------------------------------------------\n",
      "1. Text Encoding:\n",
      "   Text input_ids shape: torch.Size([3, 7])\n",
      "   Text attention_mask shape: torch.Size([3, 7])\n",
      "   Text model hidden states shape: torch.Size([3, 7, 512])\n",
      "   Text model pooler output shape: torch.Size([3, 512])\n",
      "   Text embeddings (after projection) shape: torch.Size([3, 512])\n",
      "\n",
      "2. Vision Encoding:\n",
      "   Vision pixel_values shape: torch.Size([1, 3, 224, 224])\n",
      "   Vision model hidden states shape: torch.Size([1, 50, 768])\n",
      "   Vision model pooler output shape: torch.Size([1, 768])\n",
      "   Image embeddings (after projection) shape: torch.Size([1, 512])\n",
      "\n",
      "3. Complete Forward Pass:\n",
      "   Final text features shape: torch.Size([3, 512])\n",
      "   Final image features shape: torch.Size([1, 512])\n",
      "   Logits per image shape: torch.Size([1, 3])\n",
      "   Logits per text shape: torch.Size([3, 1])\n",
      "\n",
      "4. Similarity Computation Details:\n",
      "   Image-to-text logits: torch.Size([1, 3])\n",
      "   Text-to-image logits: torch.Size([3, 1])\n",
      "   Logit scale parameter: 100.0000\n",
      "   Image-to-text probabilities shape: torch.Size([1, 3])\n",
      "   Text-to-image probabilities shape: torch.Size([3, 1])\n",
      "\n",
      "5. Detailed Results:\n",
      "------------------------------\n",
      "Raw similarity scores (logits):\n",
      "   'a photo of a cat': 19.1678\n",
      "   'a photo of a dog': 19.4523\n",
      "   'a photo of a girl': 25.7534\n",
      "\n",
      "Probability distribution:\n",
      "   'a photo of a cat': 0.0014 (0.14%)\n",
      "   'a photo of a dog': 0.0018 (0.18%)\n",
      "   'a photo of a girl': 0.9968 (99.68%)\n",
      "\n",
      "Best match: 'a photo of a girl' with probability 0.9968\n",
      "\n",
      "6. Model Parameter Statistics:\n",
      "----------------------------------------\n",
      "Total parameters: 151,277,313\n",
      "Trainable parameters: 151,277,313\n",
      "Text model parameters: 63,165,952\n",
      "Vision model parameters: 87,456,000\n",
      "Text projection parameters: 262,144\n",
      "Visual projection parameters: 393,216\n",
      "Logit scale parameter: 1\n",
      "====================================================================================================\n",
      "Analysis Complete!\n"
     ]
    }
   ],
   "execution_count": 1
  },
  {
   "cell_type": "markdown",
   "id": "ae34da64636c629c",
   "metadata": {
    "id": "ae34da64636c629c"
   },
   "source": [
    "# 下面是理解CLIP写的简单样例"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "846fd002ca4af503",
   "metadata": {
    "id": "846fd002ca4af503"
   },
   "source": [
    "相似度是在batch内计算，论文中的N就是batch size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "af1651440bafc547",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-16T12:04:59.828378Z",
     "start_time": "2025-05-16T12:04:59.785286Z"
    },
    "id": "af1651440bafc547",
    "outputId": "49dfcc3d-cf2a-4543-e582-1a0c202c6f4c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "logits_per_image: torch.Size([4, 4])\n",
      "logits_per_text: torch.Size([4, 4])\n",
      "Loss: 1.390106439590454\n"
     ]
    }
   ],
   "source": [
    "import torch  # 导入PyTorch库\n",
    "import torch.nn as nn  # 导入PyTorch神经网络模块\n",
    "import torch.nn.functional as F  # 导入PyTorch函数式API\n",
    "\n",
    "# 定义简单的图像编码器和文本编码器\n",
    "class ImageEncoder(nn.Module):  # 定义图像编码器类，继承自nn.Module\n",
    "    def __init__(self, embed_dim):  # 初始化方法，接收嵌入维度参数\n",
    "        super(ImageEncoder, self).__init__()  # 调用父类初始化方法\n",
    "        self.fc = nn.Linear(1024, embed_dim)  # 假设输入图像特征维度为 1024，创建线性层将其映射到嵌入维度\n",
    "\n",
    "    def forward(self, x):  # 前向传播方法\n",
    "        return self.fc(x)  # 返回线性变换后的结果\n",
    "\n",
    "class TextEncoder(nn.Module):  # 定义文本编码器类，继承自nn.Module\n",
    "    def __init__(self, embed_dim):  # 初始化方法，接收嵌入维度参数\n",
    "        super(TextEncoder, self).__init__()  # 调用父类初始化方法\n",
    "        self.fc = nn.Linear(512, embed_dim)  # 假设输入文本特征维度为 512，创建线性层将其映射到嵌入维度\n",
    "\n",
    "    def forward(self, x):  # 前向传播方法\n",
    "        return self.fc(x)  # 返回线性变换后的结果\n",
    "\n",
    "# 定义 CLIP 模型\n",
    "class CLIP(nn.Module):  # 定义CLIP模型类，继承自nn.Module\n",
    "    def __init__(self, embed_dim):  # 初始化方法，接收嵌入维度参数\n",
    "        super(CLIP, self).__init__()  # 调用父类初始化方法\n",
    "        self.image_encoder = ImageEncoder(embed_dim)  # 创建图像编码器实例\n",
    "        self.text_encoder = TextEncoder(embed_dim)  # 创建文本编码器实例\n",
    "        self.logit_scale = nn.Parameter(torch.ones([]) * 2.6592)  # 可学习的温度参数，初始化为2.6592\n",
    "\n",
    "    def forward(self, images, texts):  # 前向传播方法，接收图像和文本输入\n",
    "        # 编码图像和文本\n",
    "        image_features = self.image_encoder(images)  # 使用图像编码器处理图像\n",
    "        text_features = self.text_encoder(texts)  # 使用文本编码器处理文本\n",
    "\n",
    "        # 归一化特征\n",
    "        image_features = F.normalize(image_features, dim=-1)  # 对图像特征进行L2归一化\n",
    "        text_features = F.normalize(text_features, dim=-1)  # 对文本特征进行L2归一化\n",
    "\n",
    "        # 计算相似度矩阵\n",
    "        logit_scale = self.logit_scale.exp()  # 计算温度参数的指数\n",
    "        logits_per_image = logit_scale * image_features @ text_features.t()  # 计算图像-文本相似度矩阵\n",
    "        logits_per_text = logits_per_image.t()  # 计算文本-图像相似度矩阵（转置）\n",
    "\n",
    "        return logits_per_image, logits_per_text  # 返回两个相似度矩阵\n",
    "\n",
    "# 定义对比损失函数\n",
    "def clip_loss(logits_per_image, logits_per_text):  # 定义CLIP损失函数，接收两个相似度矩阵\n",
    "    # 计算图像到文本的损失\n",
    "    labels = torch.arange(logits_per_image.size(0)).to(logits_per_image.device)  # 创建标签，表示对角线位置为正样本\n",
    "    loss_image = F.cross_entropy(logits_per_image, labels)  # 计算图像到文本的交叉熵损失\n",
    "\n",
    "    # 计算文本到图像的损失\n",
    "    loss_text = F.cross_entropy(logits_per_text, labels)  # 计算文本到图像的交叉熵损失\n",
    "\n",
    "    # 总损失\n",
    "    total_loss = (loss_image + loss_text) / 2  # 计算总损失为两个方向损失的平均值\n",
    "    return total_loss  # 返回总损失\n",
    "\n",
    "# 示例数据\n",
    "batch_size = 4  # 设置批次大小为4\n",
    "embed_dim = 64  # 设置嵌入维度为64\n",
    "images = torch.randn(batch_size, 1024)  # 生成随机图像特征，维度为[4, 1024]\n",
    "texts = torch.randn(batch_size, 512)    # 生成随机文本特征，维度为[4, 512]\n",
    "\n",
    "# 初始化模型\n",
    "model = CLIP(embed_dim)  # 创建CLIP模型实例，嵌入维度为64\n",
    "\n",
    "# 前向传播\n",
    "logits_per_image, logits_per_text = model(images, texts)  # 执行模型前向传播，获取相似度矩阵\n",
    "print(\"logits_per_image:\", logits_per_image.shape)  # 打印图像-文本相似度矩阵的形状\n",
    "print(\"logits_per_text:\", logits_per_text.shape)  # 打印文本-图像相似度矩阵的形状\n",
    "# 计算损失\n",
    "loss = clip_loss(logits_per_image, logits_per_text)  # 计算对比损失\n",
    "print(\"Loss:\", loss.item())  # 打印损失值"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "96a3d712",
   "metadata": {
    "id": "96a3d712",
    "outputId": "66983278-2ac5-4352-9352-c2a0853303bb"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "logits_per_image: torch.Size([4, 4])\n",
      "logits_per_text: torch.Size([4, 4])\n",
      "Loss: 1.3869919776916504\n"
     ]
    }
   ],
   "source": [
    "import math  # 导入数学库，用于数学计算如平方根等\n",
    "import torch  # 导入PyTorch库，用于深度学习模型构建和训练\n",
    "import torch.nn as nn  # 导入PyTorch神经网络模块，提供构建神经网络的各种组件\n",
    "import torch.nn.functional as F  # 导入PyTorch函数式API，提供各种激活函数和损失函数等\n",
    "\n",
    "class PatchEmbedding(nn.Module):  # 定义图像分块嵌入类，将图像分成小块并嵌入到向量空间\n",
    "    def __init__(self, image_size=224, patch_size=16, in_channels=3, embed_dim=768):  # 初始化方法，设置默认参数\n",
    "        super().__init__()  # 调用父类初始化方法\n",
    "        self.image_size = image_size  # 存储图像大小，默认为224x224像素\n",
    "        self.patch_size = patch_size  # 存储patch的大小，用于将图像分割成固定大小的patch，patch_size通常为16，表示将图像分割成16x16大小的小块\n",
    "        self.num_patches = (image_size // patch_size) ** 2  # 计算patch的总数量，等于(图像大小/patch大小)的平方\n",
    "\n",
    "        # 将图像分成patch并进行线性投影，使用卷积层实现，卷积核大小和步长都等于patch_size\n",
    "        self.projection = nn.Conv2d(in_channels, embed_dim, kernel_size=patch_size, stride=patch_size)\n",
    "\n",
    "        # 位置编码相关参数\n",
    "        self.cls_token = nn.Parameter(torch.randn(1, 1, embed_dim))  # 创建可学习的分类token，用于整体图像表示\n",
    "        self.pos_embed = nn.Parameter(torch.randn(1, self.num_patches + 1, embed_dim))  # 创建可学习的位置编码，+1是为了包含cls_token的位置,看下面forward即可理解\n",
    "\n",
    "    def forward(self, x):  # 前向传播方法\n",
    "        B = x.shape[0]  # 获取批次大小\n",
    "        # 将图像转换为patch序列：先通过卷积层，然后将空间维度展平，最后调整维度顺序\n",
    "        x = self.projection(x).flatten(2).transpose(1, 2)\n",
    "\n",
    "        # 添加分类token：将cls_token扩展到批次维度，然后与patch序列拼接\n",
    "        cls_tokens = self.cls_token.expand(B, -1, -1)  # 扩展cls_token到批次大小\n",
    "        x = torch.cat((cls_tokens, x), dim=1)  # 在序列维度上拼接cls_token和patch序列\n",
    "\n",
    "        # 添加位置编码：将位置编码加到token序列上\n",
    "        x = x + self.pos_embed  # 添加位置编码，提供序列中位置信息\n",
    "        return x  # 返回处理后的序列\n",
    "\n",
    "class MultiHeadAttention(nn.Module):  # 定义多头注意力机制类\n",
    "    def __init__(self, embed_dim, num_heads):  # 初始化方法，接收嵌入维度和注意力头数\n",
    "        super().__init__()  # 调用父类初始化方法\n",
    "        self.num_heads = num_heads  # 存储注意力头数量\n",
    "        self.head_dim = embed_dim // num_heads  # 计算每个注意力头的维度\n",
    "        self.scale = self.head_dim ** -0.5  # 计算缩放因子，用于缩放注意力分数\n",
    "\n",
    "        self.qkv = nn.Linear(embed_dim, embed_dim * 3)  # 创建线性层，用于同时生成查询(Q)、键(K)和值(V)\n",
    "        self.proj = nn.Linear(embed_dim, embed_dim)  # 创建输出投影层，将多头注意力的结果映射回原始维度\n",
    "\n",
    "    def forward(self, x):  # 前向传播方法\n",
    "        B, N, C = x.shape  # 获取输入的批次大小、序列长度和特征维度\n",
    "        # 生成查询、键、值并重塑维度以适应多头注意力计算\n",
    "        qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, self.head_dim).permute(2, 0, 3, 1, 4)\n",
    "        q, k, v = qkv[0], qkv[1], qkv[2]  # 分离查询、键、值\n",
    "\n",
    "        # 计算注意力分数：查询和键的矩阵乘法，然后应用缩放\n",
    "        attn = (q @ k.transpose(-2, -1)) * self.scale  # 计算缩放点积注意力\n",
    "        attn = attn.softmax(dim=-1)  # 对注意力分数应用softmax归一化\n",
    "\n",
    "        # 应用注意力权重到值上，并重塑回原始维度\n",
    "        x = (attn @ v).transpose(1, 2).reshape(B, N, C)  # 计算加权和并重塑维度\n",
    "        x = self.proj(x)  # 通过输出投影层\n",
    "        return x  # 返回注意力的输出\n",
    "\n",
    "class ImageEncoder(nn.Module):  # 定义图像编码器类\n",
    "    def __init__(self, embed_dim, image_size=224, patch_size=16, num_layers=12, num_heads=12, mlp_ratio=4.0):  # 初始化方法\n",
    "        super().__init__()  # 调用父类初始化方法\n",
    "        # 创建图像分块嵌入层\n",
    "        self.patch_embed = PatchEmbedding(image_size, patch_size, 3, embed_dim)  # 创建图像分块嵌入实例\n",
    "\n",
    "        # Transformer编码器层：创建多个Transformer层\n",
    "        self.layers = nn.ModuleList([])  # 创建模块列表，用于存储Transformer层\n",
    "        for _ in range(num_layers):  # 循环创建指定数量的Transformer层\n",
    "            layer = nn.Sequential(  # 使用Sequential容器组织层的顺序\n",
    "                nn.LayerNorm(embed_dim),  # 第一个层归一化\n",
    "                MultiHeadAttention(embed_dim, num_heads),  # 多头注意力层\n",
    "                nn.LayerNorm(embed_dim),  # 第二个层归一化\n",
    "                nn.Sequential(  # MLP块，包含两个线性层和GELU激活函数\n",
    "                    nn.Linear(embed_dim, int(embed_dim * mlp_ratio)),  # 第一个线性层，扩展维度\n",
    "                    nn.GELU(),  # GELU激活函数\n",
    "                    nn.Linear(int(embed_dim * mlp_ratio), embed_dim)  # 第二个线性层，恢复维度\n",
    "                )\n",
    "            )\n",
    "            self.layers.append(layer)  # 将创建的层添加到模块列表中\n",
    "\n",
    "        self.norm = nn.LayerNorm(embed_dim)  # 最终的层归一化\n",
    "\n",
    "    def forward(self, x):  # 前向传播方法\n",
    "        x = self.patch_embed(x)  # 通过图像分块嵌入层处理输入\n",
    "\n",
    "        for layer in self.layers:  # 遍历所有Transformer层\n",
    "            x = x + layer[0](x)  # 第一个残差连接：原始输入 + 层归一化和注意力的输出\n",
    "            x = x + layer[2](layer[3](layer[1](x)))  # 第二个残差连接：中间结果 + 层归一化和MLP的输出\n",
    "\n",
    "        x = self.norm(x)  # 应用最终的层归一化\n",
    "        return x[:, 0]  # 只返回[CLS]token的特征，用作整个图像的表示\n",
    "\n",
    "class TextEncoder(nn.Module):  # 定义文本编码器类，继承自nn.Module\n",
    "    def __init__(self, embed_dim):  # 初始化方法，接收嵌入维度参数\n",
    "        super(TextEncoder, self).__init__()  # 调用父类初始化方法\n",
    "        self.fc = nn.Linear(512, embed_dim)  # 假设输入文本特征维度为 512，创建线性层将其映射到嵌入维度\n",
    "\n",
    "    def forward(self, x):  # 前向传播方法\n",
    "        # 实现类似CLIP模型中的注意力机制\n",
    "        batch_size, seq_len = x.shape[0], x.shape[1]  # 获取批次大小和序列长度\n",
    "\n",
    "        # 假设我们有一个简单的自注意力机制\n",
    "        # 创建QKV投影\n",
    "        hidden_size = 512  # 设置隐藏层大小\n",
    "        q_proj = nn.Linear(seq_len, hidden_size).to(x.device)  # 创建查询投影层，并移动到与输入相同的设备上\n",
    "        k_proj = nn.Linear(seq_len, hidden_size).to(x.device)  # 创建键投影层\n",
    "        v_proj = nn.Linear(seq_len, hidden_size).to(x.device)  # 创建值投影层\n",
    "        out_proj = nn.Linear(hidden_size, hidden_size).to(x.device)  # 创建输出投影层\n",
    "\n",
    "        # 计算QKV\n",
    "        q = q_proj(x)  # 生成查询向量\n",
    "        k = k_proj(x)  # 生成键向量\n",
    "        v = v_proj(x)  # 生成值向量\n",
    "\n",
    "        # 计算注意力分数\n",
    "        attention_scores = torch.matmul(q, k.transpose(-2, -1)) / math.sqrt(hidden_size)  # 计算缩放点积注意力分数\n",
    "        attention_probs = F.softmax(attention_scores, dim=-1)  # 对注意力分数应用softmax归一化\n",
    "\n",
    "        # 应用注意力\n",
    "        context_layer = torch.matmul(attention_probs, v)  # 计算注意力加权和\n",
    "\n",
    "        # 输出投影\n",
    "        attention_output = out_proj(context_layer)  # 通过输出投影层\n",
    "\n",
    "        # 最后通过线性层\n",
    "        return self.fc(attention_output + x)  # 添加残差连接并返回结果\n",
    "\n",
    "# 定义 CLIP 模型\n",
    "class CLIP(nn.Module):  # 定义CLIP模型类，继承自nn.Module\n",
    "    def __init__(self, embed_dim):  # 初始化方法，接收嵌入维度参数\n",
    "        super(CLIP, self).__init__()  # 调用父类初始化方法\n",
    "        self.image_encoder = ImageEncoder(embed_dim)  # 创建图像编码器实例\n",
    "        self.text_encoder = TextEncoder(embed_dim)  # 创建文本编码器实例\n",
    "        self.logit_scale = nn.Parameter(torch.ones([]) * 2.6592)  # 可学习的温度参数，初始化为2.6592\n",
    "\n",
    "    def forward(self, images, texts):  # 前向传播方法，接收图像和文本输入\n",
    "        # 编码图像和文本\n",
    "        image_features = self.image_encoder(images)  # 使用图像编码器处理图像\n",
    "        text_features = self.text_encoder(texts)  # 使用文本编码器处理文本\n",
    "\n",
    "        # 归一化特征\n",
    "        image_features = F.normalize(image_features, dim=-1)  # 对图像特征进行L2归一化\n",
    "        text_features = F.normalize(text_features, dim=-1)  # 对文本特征进行L2归一化\n",
    "\n",
    "        # 计算相似度矩阵\n",
    "        logit_scale = self.logit_scale.exp()  # 计算温度参数的指数\n",
    "        logits_per_image = logit_scale * image_features @ text_features.t()  # 计算图像-文本相似度矩阵\n",
    "        logits_per_text = logits_per_image.t()  # 计算文本-图像相似度矩阵（转置）\n",
    "\n",
    "        return logits_per_image, logits_per_text  # 返回两个相似度矩阵\n",
    "\n",
    "# 定义对比损失函数\n",
    "def clip_loss(logits_per_image, logits_per_text):  # 定义CLIP损失函数，接收两个相似度矩阵\n",
    "    # 计算图像到文本的损失\n",
    "    labels = torch.arange(logits_per_image.size(0)).to(logits_per_image.device)  # 创建标签，表示对角线位置为正样本\n",
    "    loss_image = F.cross_entropy(logits_per_image, labels)  # 计算图像到文本的交叉熵损失\n",
    "\n",
    "    # 计算文本到图像的损失\n",
    "    loss_text = F.cross_entropy(logits_per_text, labels)  # 计算文本到图像的交叉熵损失\n",
    "\n",
    "    # 总损失\n",
    "    total_loss = (loss_image + loss_text) / 2  # 计算总损失为两个方向损失的平均值\n",
    "    return total_loss  # 返回总损失\n",
    "\n",
    "# 示例数据\n",
    "batch_size = 4  # 设置批次大小为4\n",
    "embed_dim = 768  # 设置嵌入维度为768\n",
    "images = torch.randn(batch_size, 3, 224, 224)  # 生成随机图像数据，维度为[4, 3, 224, 224]\n",
    "texts = torch.randn(batch_size, 512)    # 生成随机文本特征，维度为[4, 512]\n",
    "\n",
    "# 初始化模型\n",
    "model = CLIP(embed_dim)  # 创建CLIP模型实例，嵌入维度为768\n",
    "\n",
    "# 前向传播\n",
    "logits_per_image, logits_per_text = model(images, texts)  # 执行模型前向传播，获取相似度矩阵\n",
    "print(\"logits_per_image:\", logits_per_image.shape)  # 打印图像-文本相似度矩阵的形状\n",
    "print(\"logits_per_text:\", logits_per_text.shape)  # 打印文本-图像相似度矩阵的形状\n",
    "# 计算损失\n",
    "loss = clip_loss(logits_per_image, logits_per_text)  # 计算对比损失\n",
    "print(\"Loss:\", loss.item())  # 打印损失值"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "name": "python3",
   "language": "python"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  },
  "colab": {
   "provenance": [],
   "gpuType": "T4"
  },
  "accelerator": "GPU"
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
